private void refresh() { if (peeks.size() == 0 && !endOfStreamReached) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = systemConsumer.poll(fetchSet, SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES); for (List<IncomingMessageEnvelope> systemStreamPartitionEnvelopes : envelopes.values()) { peeks.addAll(systemStreamPartitionEnvelopes); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SamzaException(e); } } } }
private void refresh() { if (peeks.size() == 0 && !endOfStreamReached) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = systemConsumer.poll(fetchSet, SystemConsumer.BLOCK_ON_OUTSTANDING_MESSAGES); for (List<IncomingMessageEnvelope> systemStreamPartitionEnvelopes : envelopes.values()) { peeks.addAll(systemStreamPartitionEnvelopes); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SamzaException(e); } } } }
private List<IncomingMessageEnvelope> consumeRawMessages(SystemConsumer consumer, Set<SystemStreamPartition> sspsToPoll) { try { Map<SystemStreamPartition, List<IncomingMessageEnvelope>> results = consumer.poll(sspsToPoll, POLL_TIMEOUT_MS); return results.entrySet() .stream() .filter(entry -> entry.getValue().size() != 0) .map(Map.Entry::getValue) .flatMap(List::stream) .collect(Collectors.toList()); } catch (Exception e) { fail("Unable to consume messages"); } return new ArrayList<>(); }
/** * Create a new {@link SystemConsumer} that returns a list of messages sequentially at each subsequent poll. * * @param pollOutputs a list of poll outputs to be returned at subsequent polls. * The i'th call to consumer.poll() will return the list at pollOutputs[i] * @return the consumer */ private SystemConsumer newConsumer(List<List<IncomingMessageEnvelope>> pollOutputs) throws Exception { SystemConsumer mockConsumer = mock(SystemConsumer.class); OngoingStubbing<Map> when = when(mockConsumer.poll(anySet(), anyLong())); for (List<IncomingMessageEnvelope> pollOutput : pollOutputs) { when = when.thenReturn(ImmutableMap.of(CHECKPOINT_SSP, pollOutput)); } when.thenReturn(ImmutableMap.of()); return mockConsumer; }
@Test public void testSinglePartitionConsumptionHappyPath() throws Exception { int partitionId = 0; TestMetricsRegistry testMetrics = new TestMetricsRegistry(); SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(partitionId)); Config eventHubConfig = createEventHubConfig(); EventHubSystemFactory factory = new EventHubSystemFactory(); SystemConsumer consumer = factory.getConsumer(SYSTEM_NAME, eventHubConfig, testMetrics); consumer.register(ssp, EventHubSystemConsumer.START_OF_STREAM); consumer.start(); int numEvents = 0; int numRetries = 20; while (numRetries-- > 0) { List<IncomingMessageEnvelope> result = consumer.poll(Collections.singleton(ssp), 2000).get(ssp); numEvents = result == null ? 0 : result.size(); if (numEvents > 0) { EventHubIncomingMessageEnvelope eventData = (EventHubIncomingMessageEnvelope) result.get(0); System.out.println("System properties: " + eventData.getEventData().getSystemProperties()); System.out.println("Key: " + new String((byte[]) eventData.getKey())); System.out.println("Message: " + new String((byte[]) eventData.getMessage())); } System.out.println("Retries left: " + numRetries); } Assert.assertTrue(numEvents > 0); }
when(mockConsumer.poll(anyObject(), anyLong())).thenReturn(sspMap);
/** * Verify that if a particular key-value is written, then another, then the original again, * that the original occurs last in the set. */ @Test public void testOrderKeyRewrite() throws InterruptedException { final SystemStream systemStream = new SystemStream("system", "stream"); final SystemStreamPartition ssp = new SystemStreamPartition(systemStream, new Partition(0)); final SystemConsumer systemConsumer = mock(SystemConsumer.class); final List<IncomingMessageEnvelope> list = new ArrayList<>(); SetConfig setConfig1 = new SetConfig("source", "key1", "value1"); SetConfig setConfig2 = new SetConfig("source", "key1", "value2"); SetConfig setConfig3 = new SetConfig("source", "key1", "value1"); list.add(createIncomingMessageEnvelope(setConfig1, ssp)); list.add(createIncomingMessageEnvelope(setConfig2, ssp)); list.add(createIncomingMessageEnvelope(setConfig3, ssp)); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages = new HashMap<SystemStreamPartition, List<IncomingMessageEnvelope>>() { { put(ssp, list); } }; when(systemConsumer.poll(anySet(), anyLong())).thenReturn(messages, Collections.<SystemStreamPartition, List<IncomingMessageEnvelope>>emptyMap()); CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(systemStream, systemConsumer, new SinglePartitionWithoutOffsetsSystemAdmin()); consumer.bootstrap(); Set<CoordinatorStreamMessage> bootstrappedMessages = consumer.getBoostrappedStream(); assertEquals(2, bootstrappedMessages.size()); // First message should have been removed as a duplicate CoordinatorStreamMessage[] coordinatorStreamMessages = bootstrappedMessages.toArray(new CoordinatorStreamMessage[2]); assertEquals(setConfig2, coordinatorStreamMessages[0]); assertEquals(setConfig3, coordinatorStreamMessages[1]); //Config 3 MUST be the last message, not config 2 }