public InFlightMessageTracker getTracker() { if (tracker == null) { tracker = new InFlightMessageTracker(logger); } return tracker; } }
public PublishResult complete() { if (tracker == null) { throw new IllegalStateException("Cannot complete publishing to Kafka because Publisher Lease was already closed"); } producer.flush(); try { tracker.awaitCompletion(maxAckWaitMillis); return tracker.createPublishResult(); } catch (final InterruptedException e) { logger.warn("Interrupted while waiting for an acknowledgement from Kafka; some FlowFiles may be transferred to 'failure' even though they were received by Kafka"); Thread.currentThread().interrupt(); return tracker.failOutstanding(e); } catch (final TimeoutException e) { logger.warn("Timed out while waiting for an acknowledgement from Kafka; some FlowFiles may be transferred to 'failure' even though they were received by Kafka"); return tracker.failOutstanding(e); } finally { tracker = null; } }
@Override public void onCompletion(final RecordMetadata metadata, final Exception exception) { if (exception == null) { tracker.incrementAcknowledgedCount(flowFile); } else { tracker.fail(flowFile, exception); poison(); } } });
public PublishResult failOutstanding(final Exception exception) { messageCountsByFlowFile.keySet().stream() .filter(ff -> !isComplete(ff)) .filter(ff -> !failures.containsKey(ff)) .forEach(ff -> failures.put(ff, exception)); return createPublishResult(); }
void publish(final FlowFile flowFile, final RecordSet recordSet, final RecordSetWriterFactory writerFactory, final RecordSchema schema, final String messageKeyField, final String topic) throws IOException { if (tracker == null) { tracker = new InFlightMessageTracker(); if (tracker.isFailed(flowFile)) { tracker.trackEmpty(flowFile); tracker.fail(flowFile, ttle); } catch (final SchemaNotFoundException snfe) { throw new IOException(snfe); } catch (final Exception e) { tracker.fail(flowFile, e); poison(); throw e;
void publish(final FlowFile flowFile, final InputStream flowFileContent, final byte[] messageKey, final byte[] demarcatorBytes, final String topic) throws IOException { if (tracker == null) { tracker = new InFlightMessageTracker(logger); if (demarcatorBytes == null || demarcatorBytes.length == 0) { if (flowFile.getSize() > maxMessageSize) { tracker.fail(flowFile, new TokenTooLargeException("A message in the stream exceeds the maximum allowed message size of " + maxMessageSize + " bytes.")); return; publish(flowFile, messageKey, messageContent, topic, tracker); if (tracker.isFailed(flowFile)) { tracker.fail(flowFile, ttle); tracker.fail(flowFile, e); poison(); throw e;
@Override public Exception getReasonForFailure(final FlowFile flowFile) { return getFailure(flowFile); } };
private boolean isComplete() { return messageCountsByFlowFile.keySet().stream() .allMatch(flowFile -> isComplete(flowFile)); }
@Override public int getSuccessfulMessageCount(final FlowFile flowFile) { return getAcknowledgedCount(flowFile); }
private void publish(final FlowFile flowFile, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) { final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent); producer.send(record, new Callback() { @Override public void onCompletion(final RecordMetadata metadata, final Exception exception) { if (exception == null) { tracker.incrementAcknowledgedCount(flowFile); } else { tracker.fail(flowFile, exception); poison(); } } }); tracker.incrementSentCount(flowFile); }
void fail(final FlowFile flowFile, final Exception cause) { getTracker().fail(flowFile, cause); rollback(); }
void publish(final FlowFile flowFile, final RecordSet recordSet, final RecordSetWriterFactory writerFactory, final RecordSchema schema, final String messageKeyField, final String topic) throws IOException { if (tracker == null) { tracker = new InFlightMessageTracker(logger); if (tracker.isFailed(flowFile)) { tracker.trackEmpty(flowFile); tracker.fail(flowFile, ttle); } catch (final SchemaNotFoundException snfe) { throw new IOException(snfe); } catch (final Exception e) { tracker.fail(flowFile, e); poison(); throw e;
void publish(final FlowFile flowFile, final InputStream flowFileContent, final byte[] messageKey, final byte[] demarcatorBytes, final String topic) throws IOException { if (tracker == null) { tracker = new InFlightMessageTracker(logger); if (demarcatorBytes == null || demarcatorBytes.length == 0) { if (flowFile.getSize() > maxMessageSize) { tracker.fail(flowFile, new TokenTooLargeException("A message in the stream exceeds the maximum allowed message size of " + maxMessageSize + " bytes.")); return; publish(flowFile, messageKey, messageContent, topic, tracker); if (tracker.isFailed(flowFile)) { tracker.fail(flowFile, ttle); tracker.fail(flowFile, e); poison(); throw e;
public PublishResult failOutstanding(final Exception exception) { messageCountsByFlowFile.keySet().stream() .filter(ff -> !isComplete(ff)) .filter(ff -> !failures.containsKey(ff)) .forEach(ff -> failures.put(ff, exception)); return createPublishResult(); }
@Override public Exception getReasonForFailure(final FlowFile flowFile) { return getFailure(flowFile); } };
private boolean isComplete() { return messageCountsByFlowFile.keySet().stream() .allMatch(flowFile -> isComplete(flowFile)); }
@Override public int getSuccessfulMessageCount(final FlowFile flowFile) { return getAcknowledgedCount(flowFile); }
protected void publish(final FlowFile flowFile, final byte[] messageKey, final byte[] messageContent, final String topic, final InFlightMessageTracker tracker) { final ProducerRecord<byte[], byte[]> record = new ProducerRecord<>(topic, null, messageKey, messageContent); producer.send(record, new Callback() { @Override public void onCompletion(final RecordMetadata metadata, final Exception exception) { if (exception == null) { tracker.incrementAcknowledgedCount(flowFile); } else { tracker.fail(flowFile, exception); poison(); } } }); messagesSent.incrementAndGet(); tracker.incrementSentCount(flowFile); }
void fail(final FlowFile flowFile, final Exception cause) { getTracker().fail(flowFile, cause); rollback(); }
void publish(final FlowFile flowFile, final InputStream flowFileContent, final byte[] messageKey, final byte[] demarcatorBytes, final String topic) throws IOException { if (tracker == null) { tracker = new InFlightMessageTracker(); if (demarcatorBytes == null || demarcatorBytes.length == 0) { if (flowFile.getSize() > maxMessageSize) { tracker.fail(flowFile, new TokenTooLargeException("A message in the stream exceeds the maximum allowed message size of " + maxMessageSize + " bytes.")); return; publish(flowFile, messageKey, messageContent, topic, tracker); if (tracker.isFailed(flowFile)) { tracker.trackEmpty(flowFile); } catch (final TokenTooLargeException ttle) { tracker.fail(flowFile, ttle); tracker.fail(flowFile, e); poison(); throw e;