private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Starts the underlying SystemConsumer. */ public void start() { if (isStarted) { log.info("Coordinator stream consumer already started"); return; } log.info("Starting coordinator stream system consumer."); systemConsumer.start(); systemAdmin.start(); isStarted = true; }
/** * Stops the underlying SystemConsumer. */ public void stop() { log.info("Stopping coordinator stream system consumer."); systemConsumer.stop(); systemAdmin.stop(); isStarted = false; }
@Test public void testSinglePartitionConsumptionHappyPath() throws Exception { int partitionId = 0; TestMetricsRegistry testMetrics = new TestMetricsRegistry(); SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(partitionId)); Config eventHubConfig = createEventHubConfig(); EventHubSystemFactory factory = new EventHubSystemFactory(); SystemConsumer consumer = factory.getConsumer(SYSTEM_NAME, eventHubConfig, testMetrics); consumer.register(ssp, EventHubSystemConsumer.START_OF_STREAM); consumer.start(); int numEvents = 0; int numRetries = 20; while (numRetries-- > 0) { List<IncomingMessageEnvelope> result = consumer.poll(Collections.singleton(ssp), 2000).get(ssp); numEvents = result == null ? 0 : result.size(); if (numEvents > 0) { EventHubIncomingMessageEnvelope eventData = (EventHubIncomingMessageEnvelope) result.get(0); System.out.println("System properties: " + eventData.getEventData().getSystemProperties()); System.out.println("Key: " + new String((byte[]) eventData.getKey())); System.out.println("Message: " + new String((byte[]) eventData.getMessage())); } System.out.println("Retries left: " + numRetries); } Assert.assertTrue(numEvents > 0); }
private void refresh() { if (peeks.size() == 0 && !endOfStreamReached) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = systemConsumer.poll(fetchSet, SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES); for (List<IncomingMessageEnvelope> systemStreamPartitionEnvelopes : envelopes.values()) { peeks.addAll(systemStreamPartitionEnvelopes); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SamzaException(e); } } } }
private void refresh() { if (peeks.size() == 0 && !endOfStreamReached) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = systemConsumer.poll(fetchSet, SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES); for (List<IncomingMessageEnvelope> systemStreamPartitionEnvelopes : envelopes.values()) { peeks.addAll(systemStreamPartitionEnvelopes); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SamzaException(e); } } } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Starts the underlying SystemConsumer. */ public void start() { if (isStarted) { log.info("Coordinator stream consumer already started"); return; } log.info("Starting coordinator stream system consumer."); systemConsumer.start(); systemAdmin.start(); isStarted = true; }
/** * Stops the underlying SystemConsumer. */ public void stop() { log.info("Stopping coordinator stream system consumer."); systemConsumer.stop(); systemAdmin.stop(); isStarted = false; }
private List<IncomingMessageEnvelope> consumeRawMessages(SystemConsumer consumer, Set<SystemStreamPartition> sspsToPoll) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> results = consumer.poll(sspsToPoll, POLL_TIMEOUT_MS); return results.entrySet() .stream() .filter(entry -> entry.getValue().size() != 0) .map(Map.Entry::getValue) .flatMap(List::stream) .collect(Collectors.toList()); } catch (Exception e) { fail("Unable to consume messages"); } return new ArrayList<>(); }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Starts the underlying SystemConsumer. */ public void start() { if (isStarted) { log.info("Coordinator stream consumer already started"); return; } log.info("Starting coordinator stream system consumer."); systemConsumer.start(); systemAdmin.start(); isStarted = true; }
/** * Stops the underlying SystemConsumer. */ public void stop() { log.info("Stopping coordinator stream system consumer."); systemConsumer.stop(); systemAdmin.stop(); isStarted = false; }
/** * Create a new {@link SystemConsumer} that returns a list of messages sequentially at each subsequent poll. * * @param pollOutputs a list of poll outputs to be returned at subsequent polls. * The i'th call to consumer.poll() will return the list at pollOutputs[i] * @return the consumer */ private SystemConsumer newConsumer(List<List<IncomingMessageEnvelope>> pollOutputs) throws Exception { SystemConsumer mockConsumer = mock(SystemConsumer.class); OngoingStubbing<Map> when = when(mockConsumer.poll(anySet(), anyLong())); for (List<IncomingMessageEnvelope> pollOutput : pollOutputs) { when = when.thenReturn(ImmutableMap.of(CHECKPOINT_SSP, pollOutput)); } when.thenReturn(ImmutableMap.of()); return mockConsumer; }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
/** * Starts the underlying SystemConsumer. */ public void start() { if (isStarted) { log.info("Coordinator stream consumer already started"); return; } log.info("Starting coordinator stream system consumer."); systemConsumer.start(); systemAdmin.start(); isStarted = true; }
/** * Stops the underlying SystemConsumer. */ public void stop() { log.info("Stopping coordinator stream system consumer."); systemConsumer.stop(); systemAdmin.stop(); isStarted = false; }
when(mockConsumer.poll(anyObject(), anyLong())).thenReturn(sspMap);