synchronized void free(SystemStreamPartition ssp) { boolean success = availableSsps.computeIfAbsent(ssp.getStream(), p -> new HashSet<>()).add(ssp); Validate.isTrue(success, String.format("Ssp %s is already in free pool.", ssp)); LOG.info("Number of unassigned partitions for system-stream {} is {}.", ssp.getSystemStream(), availableSsps.get(ssp.getStream()).size()); } }
@Override public void register(SystemStreamPartition ssp, String offset) { LOG.info("Register called with ssp {} and offset {}. Offset will be ignored.", ssp, offset); String stream = ssp.getStream(); streams.add(stream); sspAllocator.free(ssp); super.register(ssp, offset); }
private List<String> getPartitionDescriptor(SystemStreamPartition systemStreamPartition) { String streamName = systemStreamPartition.getStream(); Partition partition = systemStreamPartition.getPartition(); try { return cachedPartitionDescriptorMap.get(streamName).get(partition); } catch (ExecutionException e) { throw new SamzaException("Failed to obtain descriptor for " + systemStreamPartition, e); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
private String serializeSystemStreamPartition(SystemStreamPartition ssp) { // Create the Json string for SystemStreamPartition Map<String, String> sspMap = new HashMap<>(); sspMap.put(SYSTEM_PROP_NAME, ssp.getSystem()); sspMap.put(STREAM_PROP_NAME, ssp.getStream()); sspMap.put(PARTITION_PROP_NAME, String.valueOf(ssp.getPartition().getPartitionId())); return new String(jsonSerde.toBytes(sspMap)); }
/** * Constructs a Samza stream partition object based upon an existing Samza stream partition. * @param other Reference to an already existing Samza stream partition. */ public SystemStreamPartition(SystemStreamPartition other) { this(other.getSystem(), other.getStream(), other.getPartition()); }
@SuppressWarnings("unchecked") private boolean isSspAvailable(SSPAllocator sspAllocator, SystemStreamPartition ssp) throws NoSuchFieldException, IllegalAccessException { Field f = sspAllocator.getClass().getDeclaredField("availableSsps"); f.setAccessible(true); Map<String, Set<SystemStreamPartition>> availableSsps = (Map<String, Set<SystemStreamPartition>>) f.get( sspAllocator); return availableSsps.containsKey(ssp.getStream()) && availableSsps.get(ssp.getStream()).contains(ssp); } }
@Override public void onReceiveRecords(SystemStreamPartition ssp, List<Record> records, long millisBehindLatest) { metrics.updateMillisBehindLatest(ssp.getStream(), millisBehindLatest); records.forEach(record -> put(ssp, translate(ssp, record))); }
void setAggregateTime(SystemStreamPartition systemStreamPartition, long time) { final Gauge<Long> aggregate = aggregates.computeIfAbsent(systemStreamPartition, ssp -> newGauge(String.format("%s-%s-aggr-watermark", ssp.getStream(), ssp.getPartition().getPartitionId()), 0L)); aggregate.set(time); } }
void setAggregateTime(SystemStreamPartition systemStreamPartition, long time) { final Gauge<Long> aggregate = aggregates.computeIfAbsent(systemStreamPartition, ssp -> newGauge(String.format("%s-%s-aggr-watermark", ssp.getStream(), ssp.getPartition().getPartitionId()), 0L)); aggregate.set(time); } }
void setAggregateTime(SystemStreamPartition systemStreamPartition, long time) { final Gauge<Long> aggregate = aggregates.computeIfAbsent(systemStreamPartition, ssp -> newGauge(String.format("%s-%s-aggr-watermark", ssp.getStream(), ssp.getPartition().getPartitionId()), 0L)); aggregate.set(time); } }
@SuppressWarnings("unchecked") private boolean isSspAvailable(KinesisSystemConsumer consumer, SystemStreamPartition ssp) throws NoSuchFieldException, IllegalAccessException { SSPAllocator sspAllocator = getSspAllocator(consumer); Field f = sspAllocator.getClass().getDeclaredField("availableSsps"); f.setAccessible(true); Map<String, Set<SystemStreamPartition>> availableSsps = (Map<String, Set<SystemStreamPartition>>) f.get( sspAllocator); return availableSsps.containsKey(ssp.getStream()) && availableSsps.get(ssp.getStream()).contains(ssp); }
public Partition(SystemStreamPartition systemStreamPartition) { this(systemStreamPartition.getSystem(), systemStreamPartition.getStream(), systemStreamPartition.getPartition().getPartitionId()); }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
private IncomingMessageEnvelope translate(SystemStreamPartition ssp, Record record) { String shardId = processors.get(ssp).getShardId(); byte[] payload = new byte[record.getData().remaining()]; metrics.updateMetrics(ssp.getStream(), record); record.getData().get(payload); KinesisSystemConsumerOffset offset = new KinesisSystemConsumerOffset(shardId, record.getSequenceNumber()); return new KinesisIncomingMessageEnvelope(ssp, offset.toString(), record.getPartitionKey(), payload, shardId, record.getSequenceNumber(), record.getApproximateArrivalTimestamp()); }