@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> ssps, long timeout) throws InterruptedException { if (callbackException != null) { throw new SamzaException(callbackException); } return super.poll(ssps, timeout); }
@Override public void run() { try { // Should trigger a take() call. map.poll(FETCH, -1); } catch (InterruptedException e) { throw new RuntimeException(e); } } });
@Override public void run() { try { // Should trigger a poll(100, TimeUnit.MILLISECONDS) call. map.poll(FETCH, 100); } catch (InterruptedException e) { throw new RuntimeException(e); } } });
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { if (blockpollFlag) { blockConsumerPoll.await(); } return super.poll(systemStreamPartitions, timeout); }
/** * {@inheritDoc} */ @Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { systemStreamPartitions.forEach(systemStreamPartition -> { Future status = readerRunnableStatus.get(systemStreamPartition); if (status.isDone()) { try { status.get(); } catch (ExecutionException | InterruptedException e) { MultiFileHdfsReader reader = readers.get(systemStreamPartition); LOG.warn( String.format("Detect failure in ReaderRunnable for ssp: %s. Try to reconnect now.", systemStreamPartition), e); reader.reconnect(); readerRunnableStatus.put(systemStreamPartition, executorService.submit(new ReaderRunnable(reader))); } } }); return super.poll(systemStreamPartitions, timeout); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { LOG.info("{}: KafkaConsumerProxy is not running. Stopping the consumer.", this); stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { // check if the proxy is running if (!proxy.isRunning()) { stop(); String message = String.format("%s: KafkaConsumerProxy has stopped.", this); throw new SamzaException(message, proxy.getFailureCause()); } return super.poll(systemStreamPartitions, timeout); }
@Override public Map<SystemStreamPartition, List<IncomingMessageEnvelope>> poll( Set<SystemStreamPartition> systemStreamPartitions, long timeout) throws InterruptedException { Throwable handlerError = eventHubNonTransientError.get(); /* * We will retry for non transient error by instantiating a new EventHubs client if * 1. Last retry happened more than CONFIG_MIN_RETRY_INTERVAL_MS ms ago. Otherwise we ignore * 2. We haven't reached CONFIG_MAX_RETRY_COUNT allowed within the CONFIG_RETRY_WINDOW_MS window. * Otherwise we throw */ if (handlerError != null && clock.currentTimeMillis() - lastRetryTs > config.getMinRetryIntervalMs(systemName)) { int currentRetryCount = recentRetryAttempts.size(); long maxRetryCount = config.getMaxRetryCount(systemName); if (currentRetryCount < maxRetryCount) { LOG.warn("Received non transient error. Will retry.", handlerError); LOG.info("Current retry count within window: {}. max retry count allowed: {}. window size: {} ms", currentRetryCount, maxRetryCount, config.getRetryWindowMs(systemName)); long now = clock.currentTimeMillis(); recentRetryAttempts.update(now); lastRetryTs = now; reconnectTaskStatus = reconnectTaskRunner.submit(this::renewEventHubsClient); } else { LOG.error("Retries exhausted. Reached max allowed retries: ({}) within window {} ms", currentRetryCount, config.getRetryWindowMs(systemName)); String msg = "Received a non transient error from event hub partition receiver"; throw new SamzaException(msg, handlerError); } } return super.poll(systemStreamPartitions, timeout); }
@Test public void testShouldBlockAtLeast100Ms() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); long now = System.currentTimeMillis(); map.poll(FETCH, 100); assertTrue(System.currentTimeMillis() - now >= 100); }
@Test public void testShouldGetSomeMessages() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); map.put(SSP, ENVELOPE); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); assertEquals(1, envelopes.size()); assertEquals(1, envelopes.get(SSP).size()); map.put(SSP, ENVELOPE); map.put(SSP, ENVELOPE); envelopes = map.poll(FETCH, 0); assertEquals(1, envelopes.size()); assertEquals(2, envelopes.get(SSP).size()); // Size info. assertEquals(0, map.getMessagesSizeInQueue(SSP)); }
@Test public void testNoSizeComputation() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(); map.register(SSP, "0"); map.put(SSP, ENVELOPE); map.put(SSP, ENVELOPE); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); // Size info. assertEquals(0, map.getMessagesSizeInQueue(SSP)); }
@Test public void testSizeComputation() throws InterruptedException { BlockingEnvelopeMap map = new MockBlockingEnvelopeMap(true); map.register(SSP, "0"); map.put(SSP, ENVELOPE_WITH_SIZE); map.put(SSP, ENVELOPE_WITH_SIZE); // Size info. assertEquals(200, map.getMessagesSizeInQueue(SSP)); Map<SystemStreamPartition, List<IncomingMessageEnvelope>> envelopes = map.poll(FETCH, 0); assertEquals(0, map.getMessagesSizeInQueue(SSP)); }